On Debian, to remove all GUI packages, you can remove the two libraries used to connect to display servers:

apt purge libx11-6 libwayland-client0

########################################
apt-get purge task-desktop

apt-get remove -y --purge x11-common
apt-get autoremove -y --purge
apt-get install -y deborphan
deborphan | xargs dpkg -p # do this a bunch of times